import numpy as np
import csv
from datetime import datetime
from time import strftime
import math

def getCurTime():
    """
    get current time
    Return value of the date string format(%Y-%m-%d %H:%M:%S)
    """
    format='%Y-%m-%d %H:%M:%S'
    sdate = None
    cdate = datetime.now()
    try:
        sdate = cdate.strftime(format)
    except:
        raise ValueError
    return sdate

def build_data_list(inputCSV):
    sKey = []
    fn = inputCSV
    f = open(inputCSV)
    #ra = csv.DictReader(file(fn), dialect="excel")
    ra = csv.DictReader(f, dialect="excel")
    #print inputCSV
    #print ra
    
    for record in ra:
        #print record[ra.fieldnames[0]], type(record[ra.fieldnames[-1]])
        for item in ra.fieldnames:
            temp = int(float(record[item]))
            sKey.append(temp)
    sKey = np.array(sKey)
    sKey.shape=(-1,len(ra.fieldnames))
    return sKey

def distance(x1, y1, x2, y2):
    #print x1, y1, x2, y2
    x1 = float(x1)
    y1 = float(y1)
    x2 = float(x2)
    y2 = float(y2)
    temp = int(math.pow((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2), 0.5))
    return temp
    


#--------------------------------------------------------------------------
#MAIN

if __name__ == "__main__":
    print '===================================================='
    print "begin at " + getCurTime()

    filepath = 'C:/_DATA/migration_census_2000/million_pop/data2/'
    dataCSV = filepath + 'migrationflows_smoothed_2000(2).csv'
    data = build_data_list(dataCSV)
    
    excludelist = [2,15]    #2: Alaska, 15: Hawaii

    countycsv = 'C:/_DATA/migration_census_2000/county_FIPS_FID.csv'
    county = build_data_list(countycsv) #[fips, fid]

    county_id = {}

    for item in county:
        county_id[int(item[0])] = int(item[1])

    grossflowcsv = filepath + 'migration_county_summary_2000(2).csv'
    grossflow = build_data_list(grossflowcsv) #[fips, out, in]

    grossflowdata = np.zeros((len(grossflow), 2))   #[total_in, total_out]
    for item in grossflow:
        id = county_id[item[0]]
        grossflowdata[id, 0] = item[2]
        grossflowdata[id, 1] = item[1]


    formatdata = []
    # fips 8014 wasn't found in the migration file
    for item in data:
        if item[0]/1000 in excludelist or item[1]/1000 in excludelist:
            continue
        if item[0] == item[1]:
            continue
        if item[2] < 1:
            continue
        #if item[0] == 51560 or item[1] == 51560:
            #continue
        oid = county_id[item[0]]
        did = county_id[item[1]]
        #dist = distance(county[oid,2], county[oid,3], county[did,2], county[did,3])
        formatdata.append([oid, did, item[2], item[3]])
    #print formatdata
    formatdata = np.array(formatdata)
    formatdata.shape = (-1, 4)

    #grossflow = np.zeros((len(county), 2))


    regressiondata = []
    for item in formatdata:
        oid = item[0]
        did = item[1]
        #dist = distance(county[oid,2], county[oid,3], county[did,2], county[did,3])
        regressiondata.append([item[-1], grossflowdata[oid,1], grossflowdata[did,0], item[2]])
    regressiondata = np.array(regressiondata)
    regressiondata.shape = (-1, 4)
    #metroCountyCSV = filepath + 'metropolitan/metro_county_list.csv'
    #metroCounty = build_data_list(metroCountyCSV)

    #countyunique = np.unique(formatdata[:,1])
    #print countyunique, len(countyunique)
    headerstr = 'oid,did,vol'
    np.savetxt(filepath + 'million_pop_dataformat.csv', formatdata[:,[0,1,2]], delimiter=',', header = headerstr, fmt = '%s')

    headerstr = 'grossin,grossout'
    np.savetxt(filepath + 'million_pop_grossflow.csv', grossflowdata, delimiter=',', header = headerstr, fmt = '%s')

    headerstr = 'dist,grossout,grossin,vol'
    np.savetxt(filepath + 'million_pop_regressiondata.csv', regressiondata, delimiter=',', header = headerstr, fmt = '%s')


    '''
    metro_id = {}

    i = 0    
    for item in metro:
        metro_id[int(item)] = i
        i += 1

    county_id = {}

    for item in metroCounty:
        county_id[int(item[0])] = metro_id[int(item[1])]

    #print metro
    print metro_id

    
    for item in data:
        if int(item[1]) in county_id:
            item[-1] = county_id[int(item[1])]
        else:
            item[-1] = 999

    #print max(data[:,-1])



    #np.savetxt(filepath + 'metropolitan/census_county_pop_metro.csv', data, delimiter=',', fmt = '%s')
    '''